#  原始知识库html格式保存
import pandas as pd

# zsk_data_path = r'/workspace/qanything_local/qanything_kernel/row_data/poc_1.7w+_data/2024-03-29_市热线-知识库点点通信息_17603.csv'
# zsk_data_path = r'/workspace/qanything_local/qanything_kernel/row_data/poc_1.7w+_data/extral-市热线-知识库点点通信息_6276.csv'
# zsk_data_path = r'/workspace/qanything_local/qanything_kernel/row_data/poc_1.7w+_data/市热线-知识库点点通信息_17603.csv'

# import csv

# zw_dir = r'/workspace/qanything_local/ZW_HTML_FILES/2024-04-03_17K'
# # 读取CSV文件
# import csv
# import os
# import _csv

# _csv.field_size_limit(1000000)

# # 读取CSV文件
# with open(zsk_data_path, mode='r', encoding='utf-8') as csvfile:
#     reader = csv.DictReader(csvfile)
#     for row in reader:
#         # 获取id和content
#         file_id = row['\ufeff知识库ID']
#         content = row['内容']
    
#         # 以id为文件名，content为内容创建新的html文件
#         file_path = os.path.join(zw_dir, file_id)
#         with open(f'{file_path}.html', mode='w', encoding='utf-8') as htmlfile:
#             htmlfile.write(content)


# 拆分问答类数据
import pandas as pd
from html2text import html2text

zsk_data_path = r'/workspace/qanything_local/qanything_kernel/row_data/poc_1.7w+_data/为企知识库.csv'

df = pd.read_csv(zsk_data_path, encoding='utf-8')


import re

def extract_qa_pairs(text):
    # 使用正则表达式匹配任意位置的问答对
    pattern = r"([\s\S]*?问：[\s\S]*?答：[\s\S]*?(?=问：|$))"
    matches = re.findall(pattern, text)

    # 解析匹配到的问答对
    qa_pairs = []
    for match in matches:
        question = re.search(r"问：([\s\S]*?)答：", match).group(1).strip()
        question = question.strip('**\n\n**')
        answer = re.search(r"答：([\s\S]*)", match).group(1).strip()
        answer = answer.lstrip("*|\n").split('\n\n')[0]
        qa_pairs.append((question, answer))
        # print(question, answer)
        # print('\n')

    return qa_pairs

processed_rows = []

# 遍历原始DataFrame的每一行
for index, row in df.iterrows():
    # 如果内容分类是问答类
    if row['KL_CONTENT_TYPE'] == '问答类':
        try:
            qa_content = html2text(row['KL_CONTENT'])
        except Exception as e:
            print(f'第{index}个文件内容错误！')
        qa_pairs = extract_qa_pairs(qa_content)
        # 为每个问题-答案对创建一个新行
        for question, answer in qa_pairs:
            temp_row = row.copy()  # 复制当前行
            # new_row = temp_row[['知识库ID','关键字','标题']]
            new_row = temp_row[['KL_ID','KL_CONTENT_KEYWORD','KL_CONTENT_TITLE']]
            new_row['问题'] = question  # 添加问题
            new_row['答案'] = answer  # 添加答案
            processed_rows.append(new_row)  # 将新行添加到处理后的行列表中

new_df = pd.DataFrame(processed_rows)

new_df.columns = ['知识库ID', '关键字', '标题', '问题', '答案']

qa_pair_path = r'/workspace/qanything_local/qanything_kernel/row_data/poc_1.7w+_data/weiqi_qa_pair.csv'

# 将新的DataFrame写入CSV文件
new_df.to_csv(qa_pair_path, index=False)